import warnings
warnings.filterwarnings('ignore')
### helpers
import re
import numpy as np
def recode_species(species_value):
"""Takes a string and returns classified species"""
if species_value in ['Cpb','cpb', 'C.p.b.']:
return 'Cpb'
elif species_value in ['Red-eared slider', 'RES','REs']:
return 'Res'
else:
return species_value
def recode_gravid(gravid_value):
if str(gravid_value).upper() == "YES":
return True
else:
return False
def recode_sex(sex_value):
"""Takes a string and returns f, m or unknown"""
if sex_value in ['Male','male?','m','M']:
return 'm'
elif sex_value in ['Female','F','f']:
return 'f'
else:
return 'unknown'
def recode_season(date):
if date.month <= 6:
return 'spr'
else:
return 'fal'
def recode_decimal(dirty_decimal=''):
"""Takes a string and returns a decimal"""
_ = []
if not dirty_decimal:
return 0
if str(dirty_decimal):
_ = re.findall(r"[-+]?\d*\.\d+|\d+",str(dirty_decimal))
if _:
return _[0]
else:
return 0
def ecdf(data):
"""Compute ECDF for a one-dimensional array of measurements."""
# Number of data points: n
n = len(data)
# x-data for the ECDF: x
x = np.sort(data)
# y-data for the ECDF: y
y = np.arange(1, n+1) / n
return x, y
def permutation_sample(data1, data2):
"""Generate a permutation sample from two data sets."""
"""simulate the hypothesis that two variables have identical probability distributions."""
# Concatenate the data sets: data
data = np.concatenate((data1,data2))
# Permute the concatenated array: permuted_data
permuted_data = np.random.permutation(data)
# Split the permuted array into two: perm_sample_1, perm_sample_2
perm_sample_1 = permuted_data[:len(data1)]
perm_sample_2 = permuted_data[len(data1):]
return perm_sample_1, perm_sample_2
### turtels!
import pandas as pd
import numpy as np
def get_clean_data():
fileName1 = 'source/Turtle Data.xls'
fileName2 = 'source/MF Trapping Data.xlsx'
df = clean_data(fileName1,True)
df['Capture Location'] = 'Gresham'
df2 = clean_data(fileName2)
df2['Capture Location'] = 'Mason Flats'
df2['Source'] = 'MF Trapping Data.xlsx|All Capture Data'
df = df.append(df2,sort=False)
return df
def clean_data(fileName,big_file=False):
#columnNames = ['Name','Address']
print ("Loading data " + fileName)
df = pd.DataFrame()
if (big_file):
for year in range(2008,2014+1):
print(year)
new = pd.read_excel(fileName,sheet_name=str(year))
new['Source'] = '{}|{}'.format('Turtle Data.xls',str(year))
df = df.append(new,sort=False)
else:
df = pd.read_excel(fileName)
#DATA CLEANING
cleaned = df.copy()
# decimals
print ("Cleaning decimals ...")
cleaned['Weight'] = cleaned['Weight'].apply(recode_decimal)
cleaned['Weight'] = pd.to_numeric(cleaned['Weight'],downcast='float')
cleaned['Carapace'] = cleaned['Carapace'].apply(recode_decimal)
cleaned['Carapace'] = pd.to_numeric(cleaned['Carapace'],downcast='float')
cleaned['Plastron'] = cleaned['Plastron'].apply(recode_decimal)
cleaned['Plastron'] = pd.to_numeric(cleaned['Plastron'],downcast='float')
cleaned['Annuli'] = cleaned['Annuli'].apply(recode_decimal)
cleaned['Annuli'] = pd.to_numeric(cleaned['Annuli'],downcast='integer')
# other
print ("Cleaning other values ...")
cleaned['Gender'] = cleaned['Gender'].apply(recode_sex)
cleaned['Species'] = cleaned['Species'].apply(recode_species)
cleaned['Gravid'] = cleaned['Gravid'].apply(recode_gravid)
# add features
cleaned['Age_To_Weight'] = cleaned['Annuli'] / cleaned['Weight']
buckets = 5
buckets = int(cleaned['Annuli'].max() / buckets)
labels = ["{0} - {1}".format(i, i + buckets) for i in range(0, cleaned['Annuli'].max(), buckets)]
cleaned['Annuli_Group'] = pd.cut(cleaned.Annuli, range(0, cleaned.Annuli.max()+buckets, buckets), right=False, labels=labels)
# Calcuate Number of recaptures
df = cleaned[['ID','Date']].groupby('ID').count()
df.columns = ['recapture_count']
df.reset_index(inplace=True)
cleaned = pd.merge(cleaned,df,how='outer',on='ID')
# recalculate annuli
df = pd.pivot_table(cleaned[cleaned.Annuli > 0],values=['Date','Annuli'],index=['ID'],aggfunc={'Date': min,'Annuli': min})
df.columns = ['lowest_annuli','first_date']
df.reset_index(inplace=True)
cleaned = pd.merge(cleaned,df,how='outer',on='ID')
cleaned['date_year'] = cleaned.Date.map(lambda x: x.year)
cleaned['first_date_year'] = cleaned.first_date.map(lambda x: x.year)
cleaned['new_annuli'] = cleaned.date_year - cleaned.first_date_year + cleaned.lowest_annuli
cleaned.new_annuli = np.nan_to_num(cleaned.new_annuli)
# distinguish Spring, Fall and pregnant females (don't care about juvenilles/unknown)
cleaned['gender_plus'] = cleaned['Gender']
#cleaned.loc[cleaned.gender_plus != 'unknown','gender_plus'] = cleaned.Gender + '_' + cleaned.Date.apply(hlp.recode_season)
cleaned.loc[cleaned.Gravid == True,'gender_plus'] = 'f_gra'
cleaned['gender_seasons'] = cleaned['Gender']
cleaned.loc[cleaned.gender_seasons != 'unknown','gender_seasons'] = cleaned.Gender + '_' + cleaned.Date.apply(recode_season)
cleaned.loc[cleaned.Gravid == True,'gender_seasons'] = 'f_gra'
return cleaned
